use utf8;
use strict;
use Lingua::RU::OpenCorpora::Tokenizer;

binmode(STDIN, ":encoding(utf-8)");
binmode(STDOUT, ":encoding(utf-8)");
binmode(STDERR, ":encoding(utf-8)");

my %dict;

load_dict("dic/dict.opcorpora.txt");

my $tokenizer = Lingua::RU::OpenCorpora::Tokenizer->new;
my %XmlEntries = ('&' => 'amp', '"' => 'quot', "'" => 'apos', "<" => 'lt', ">" => 'gt');

my $l = 1;

my @lines;

my $title = <STDIN>;

#my $tokens = $tokenizer->tokens($title, {threshold => 0.4});
#foreach my $t (@{$tokens}) {
#  print $t . " " . lemmatize($t) . "\n";
#}
#die;

while (<STDIN>) {

  chomp $_;
  if (length($_) < 2) {
    next;
  }
 
#  $_ =~ s/<img src=""data:image\/png;base64.*$//;

  if ($_ !~ /^[0-9]+\t/) {
    $lines[$#lines] .= $_;
    #die "at line $l";
  } else {
    push @lines, $_;
  }
  $l += 1;
}

my %data;
my %textByOwner;
my %ownerName;
my @allText;

#print $title;
#print scalar @lines;
foreach my $line (@lines) {
  my ($id, $parentid, $mainideaid, $problemid, $dt, $ownerid, $ownername, @other ) = split(/\t/, $line);

#  if (!($id =~ /\d/)) { next; }

  my $text = join(" ", @other);
  save_text($id, \$text, "dump");

  $dt =~ s/^\"(\d\d\d\d)\-(\d\d)\-(\d\d)\s(\d\d):(\d\d).*\"/$3.$2.$1 $4:$5/;

  remove_tags(\$text);

  $ownername =~ s/^\"(.*)\"$/$1/;
  $text =~ s/^\"(.*)\"$/$1/;
  $text =~ s/&nbsp;/ /g;

  if (exists $data{$id}) {
    die "duplicate $id";
  }

  $data{$id}->{parent} = $parentid;
  $data{$id}->{mainidea} = $mainideaid;
  $data{$id}->{problem} = $problemid;
  $data{$id}->{datetime} = $dt;
  $data{$id}->{owner} = $ownerid;
  $data{$id}->{ownername} = $ownername;
  $data{$id}->{text} = $text;

#  print $id . "\n";

  push @{$textByOwner{$ownerid}}, $id;
  push @allText, $id;

  if (! exists $ownerName{$ownerid}) {
    $ownerName{$ownerid} = $ownername;
  }
 
# if (334 == $ownerid) {
#   print join("\t", ($id, $parentid, $mainideaid, $problemid, $dt, $ownerid, $ownername, $text)) . "\n"; 
# }
}


#print STDERR scalar keys %textByOwner;

open(F, "> fulltext.xml") || die "can't open fulltext.xml";
binmode(F, ":encoding(utf-8)");
print F serialize_to_oc_all();
close(F);
#die "nothing more";

foreach my $o (keys %textByOwner) {
  my $n = scalar @{$textByOwner{$o}};
  if ($n >= 0) {
    #print STDERR join("\t", ($o, $ownerName{$o}, $n, length($textByOwner{$o}))) . "\n";
    my $fn = "peruser/user$o.xml";
    print $fn . "\n";
    open(U, "> $fn") || die "can't open $fn";
    binmode(U, ":encoding(utf-8)");
    print U serialize_to_oc($o);
    close(U);
  }
}

sub remove_tags {
  my $r = shift;

  if ($$r =~ s/<\!\-\-\[if gte mso 9\]><xml>(.*?)<\!\[endif\]\-\->//g) {
#    print STDERR "1 " . $1 . "\n";
  }
  if ($$r =~ s/{mso([^\}]+?)}//g) {
#    print STDERR "2 " . $1 . "\n";
  }
  if ($$r =~ s/<style>(.*?)<\/style>//g) {
#    print STDERR "3 " . $1 . "\n";
  }
  if ($$r =~ s/<([^>]+?)>//g) {
#    print STDERR "4 " . $1 . "\n";
  }
  if ($$r =~ s/<img src=""data:image\/png;base64(.*)$//) {
#    print STDERR "5 " . $1 . "\n";
  }
  if ($$r =~ s/<([A-Za-z][^А-ЯЁа-яё]*)$//g) {
#    print STDERR "6 " . $1 . "\n";
  }
#  $$r =~ s/{[^А-ЯЁа-яё]+}//g;
}

sub serialize_to_oc {
  my $oid = shift;
  my $t;

  $t = "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\"?>\n";
  $t .= "<annotation version=\"0.1\" revision=\"1\">\n";

  my ($parid, $sentid, $tid) = (0, 0, 0);
  my @a = ();
  foreach my $id (@{$textByOwner{$oid}}) {
    $t .= make_xml_text($data{$id}->{parent}, $id, "", \@a, \$data{$id}->{text}, \$parid, \$sentid, \$tid);
  }

#  my $ctrl_text;
#  for(my $i = 0; $i < ($tid / 1000); $i++) {
#    $ctrl_text .= " " . "контрольноесловонатысячу";
#  }

#  $t .= make_xml_text(55559999, 66669999, "", \@a, \$ctrl_text, \$parid, \$sentid, \$tid);

  $t .= "</annotation>\n";

  return $t;
}

sub serialize_to_oc_all {
  my $t;

  $t = "<?xml version=\"1.0\" encoding=\"utf-8\" standalone=\"yes\"?>\n";
  $t .= "<annotation version=\"0.1\" revision=\"1\">\n";
  my @a = ();

  my ($parid, $sentid, $tid) = (0, 0, 0);
  foreach my $id (keys %data) {
    $t .= make_xml_text($data{$id}->{parent}, $id, "", \@a, \$data{$id}->{text}, \$parid, \$sentid, \$tid);
    #save_text($id, \$data{$id}->{text}, "dump.post");
  }

#  my $ctrl_text;
#  for(my $i = 0; $i < ($tid / 1000); $i++) {
#    $ctrl_text .= " " . "контрольноесловонатысячу";
#  }

#  $t .= make_xml_text(55559999, 66669999, "", \@a, \$ctrl_text, \$parid, \$sentid, \$tid);

  $t .= "</annotation>\n";

print STDERR length($t) . "\n";

  return $t;
}

sub make_xml_text {
  my ($pid, $id, $name, $rat, $rt, $parid, $sentid, $tid) = @_;
  my $x;

  $x .= "<text id=\"$id\" parent=\"$pid\" name=\"$name\">\n";
  $x .= "  <tags>\n";
  foreach my $t (@{$rat}) {
    $x .= "    <tag>$t</tag>\n";
  }
  $x .= "  </tags>\n";
  
  $x .= "  <paragraphs>\n";
  my @par = split(/\n+/, $$rt);
  foreach my $p (@par) {
    $x .= "    <paragraph id=\"$$parid\">\n";
    $p =~ s/([\.\!\?])\s+([A-ZА-ЯЁ])/$1 SENTBREAK $2/sg;
    my @sent = split(/ SENTBREAK /, $p);
    foreach my $s (@sent) {
      $s =~ s/^\s+//;
      $s =~ s/\s+$//;
      $s = encode_xml_char($s);
      $x .= "      <sentence id=\"$$sentid\">\n";
      $x .= "        <source>$s</source>\n";
      
      #do tokenization
      $x .= "        <tokens>\n";

      my $tokens = $tokenizer->tokens($s, {threshold => 0.6});
      foreach my $t (@{$tokens}) {
        $t = encode_xml_char($t);
        my $l = lemmatize($t);
        $x .= "          <token id=\"$$tid\" text=\"$t\">";
        $x .= "<tfr rev_id=\"1\" t=\"$t\"><v><l id=\"0\" t=\"$l\"><g v=\"UNKN\"/></l></v></tfr></token>\n";
        $$tid += 1;
      }

      $x .= "        </tokens>\n";
      $x .= "      </sentence>\n";
      $$sentid += 1;
    }

    $x .= "    </paragraph>\n";
    $$parid += 1;
  } 
  $x .= "  </paragraphs>\n";
  $x .= "</text>\n";

  return $x;
}

sub encode_xml_char {
  my $s = shift;
  $s =~ s/([&"'<>])/&$XmlEntries{$1};/g;
  return $s;
}

sub lemmatize {
  my $w = lc(shift);
  $w =~ tr/[Ёё]/[Ее]/;

  #print STDERR "LEM $w\n";

  if (exists $dict{$w}) {
    my @a = keys %{$dict{$w}};
    if ($#a > 0) {
      #print STDERR "AMB $w: " . join(" ", @a) . "\n";
    }
    return $a[0];
  } elsif ($w =~ /^([^\-]+)\-([^\-]+)$/) {
    my ($p1, $p2) = ($1, $2);
    my $p2l = lemmatize($p2);
    return $p1 . "-" . $p2l;
  }
  return $w;
}

sub load_dict {
  my $fn = shift;
  my $l = undef;

  open(F, "< $fn") || die "can't open $fn";
  binmode(F, ":encoding(utf-8)");
  while (<F>) {
    chomp $_;
    if ($_ =~ /^\d+/) {
      $l = undef;
      next;
    } elsif ($_ =~ /^([А-ЯЁа-яё\-]+)/) {
      my $f = lc($1);
      $f =~ tr/[Ёё]/[Ее]/;
      if (!defined($l)) {
        $l = $1;
      }
      $dict{$f}->{$l} += 1;
    }
  }
  close(F);
}

sub save_text {
  my ($id, $rt, $dir) = @_;
  my $fn = "$dir\/t$id.txt";
  open(F, "> $fn") || die "can't open $fn";
  binmode(F, ":encoding(utf-8)");
  print F $$rt;
  close(F);
}
